<!doctype html>



  


<html class="theme-next mist use-motion" lang="zh-Hans">
<head>
  <!-- hexo-inject:begin --><!-- hexo-inject:end --><meta charset="UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>



<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />












  
  
  <link href="/lib/fancybox/source/jquery.fancybox.css?v=2.1.5" rel="stylesheet" type="text/css" />




  
  
  
  

  
    
    
  

  

  

  

  

  
    
    
    <link href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic&subset=latin,latin-ext" rel="stylesheet" type="text/css">
  






<link href="/lib/font-awesome/css/font-awesome.min.css?v=4.6.2" rel="stylesheet" type="text/css" />

<link href="/css/main.css?v=5.1.0" rel="stylesheet" type="text/css" />


  <meta name="keywords" content="机器学习,逻辑回归," />








  <link rel="shortcut icon" type="image/x-icon" href="http://okqpu7sb4.bkt.clouddn.com/image/user/logo.png?v=5.1.0" />






<meta name="description" content="&amp;emsp;&amp;emsp;回归和分类是机器学习解决的最重要的问题，本篇主要整理了线性回归和逻辑回归的基本推理过程和工程实践应用，首先对两种回归的方法做一下比较:

线性回归: 最小二乘法推导，预测连续值
逻辑回归: 基于最大似然估计，主要解决二分类问题，也可用于连续值的预测">
<meta property="og:type" content="article">
<meta property="og:title" content="逻辑回归与线性回归原理探微">
<meta property="og:url" content="http://onesimple.cn/blog/2017/02/logistic-regression/index.html">
<meta property="og:site_name" content="onesimple">
<meta property="og:description" content="&amp;emsp;&amp;emsp;回归和分类是机器学习解决的最重要的问题，本篇主要整理了线性回归和逻辑回归的基本推理过程和工程实践应用，首先对两种回归的方法做一下比较:

线性回归: 最小二乘法推导，预测连续值
逻辑回归: 基于最大似然估计，主要解决二分类问题，也可用于连续值的预测">
<meta property="og:image" content="http://okqpu7sb4.bkt.clouddn.com//image/cover/ml.jpg?imageView2/2/w/600">
<meta property="og:image" content="http://okqpu7sb4.bkt.clouddn.com//image/data/line-rg1.png?imageView2/2/w/600">
<meta property="og:image" content="http://okqpu7sb4.bkt.clouddn.com//image/data/sigmoid.jpg?imageView2/2/w/600">
<meta property="og:updated_time" content="2018-12-11T15:21:09.071Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="逻辑回归与线性回归原理探微">
<meta name="twitter:description" content="&amp;emsp;&amp;emsp;回归和分类是机器学习解决的最重要的问题，本篇主要整理了线性回归和逻辑回归的基本推理过程和工程实践应用，首先对两种回归的方法做一下比较:

线性回归: 最小二乘法推导，预测连续值
逻辑回归: 基于最大似然估计，主要解决二分类问题，也可用于连续值的预测">
<meta name="twitter:image" content="http://okqpu7sb4.bkt.clouddn.com//image/cover/ml.jpg?imageView2/2/w/600">



<script type="text/javascript" id="hexo.configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Mist',
    sidebar: {"position":"left","display":"post"},
    fancybox: true,
    motion: true,
    duoshuo: {
      userId: 'undefined',
      author: '博主'
    },
    algolia: {
      applicationID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"输入关键字","hits_empty":"没有找到与 ${query} 相关的内容","hits_stats":"${hits} 条相关记录，共耗时 ${time} ms"}
    }
  };
</script>



  <link rel="canonical" href="http://onesimple.cn/blog/2017/02/logistic-regression/"/>





  <title> 逻辑回归与线性回归原理探微 | onesimple </title><!-- hexo-inject:begin --><!-- hexo-inject:end -->
</head>

<body itemscope itemtype="http://schema.org/WebPage" lang="zh-Hans">

  



  <!-- hexo-inject:begin --><!-- hexo-inject:end --><script type="text/javascript">
    var _hmt = _hmt || [];
    (function() {
      var hm = document.createElement("script");
      hm.src = "https://hm.baidu.com/hm.js?d6c28a35566fe8266f750f1cb6a16433";
      var s = document.getElementsByTagName("script")[0];
      s.parentNode.insertBefore(hm, s);
    })();
  </script>








  
  
    
  

  <div class="container one-collumn sidebar-position-left page-post-detail ">
    <div class="headband"></div>

    <header id="header" class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-meta ">
  

  <div class="custom-logo-site-title">
    <a href="/"  class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <span class="site-title">onesimple</span>
      <span class="logo-line-after"><i></i></span>
    </a>
  </div>
    
      <p class="site-subtitle"></p>
    
</div>

<div class="site-nav-toggle">
  <button>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
    <span class="btn-bar"></span>
  </button>
</div>

<nav class="site-nav">
  

  
    <ul id="menu" class="menu">
      
        
        <li class="menu-item menu-item-home">
          <a href="/" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-home"></i> <br />
            
            首页
          </a>
        </li>
      
        
        <li class="menu-item menu-item-archives">
          <a href="/archives" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-archive"></i> <br />
            
            归档
          </a>
        </li>
      
        
        <li class="menu-item menu-item-tags">
          <a href="/tags" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-tags"></i> <br />
            
            标签
          </a>
        </li>
      
        
        <li class="menu-item menu-item-about">
          <a href="/about" rel="section">
            
              <i class="menu-item-icon fa fa-fw fa-user"></i> <br />
            
            关于
          </a>
        </li>
      

      
        <li class="menu-item menu-item-search">
          
            <a href="javascript:;" class="popup-trigger">
          
            
              <i class="menu-item-icon fa fa-search fa-fw"></i> <br />
            
            搜索
          </a>
        </li>
      
    </ul>
  

  
    <div class="site-search">
      
  <div class="popup">
 <span class="search-icon fa fa-search"></span>
 <input type="text" id="local-search-input">
 <div id="local-search-result"></div>
 <span class="popup-btn-close">close</span>
</div>


    </div>
  
</nav>



 </div>
    </header>

    <main id="main" class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div id="content" class="content">
            

  <div id="posts" class="posts-expand">
    

  

  
  
  

  <article class="post post-type-normal " itemscope itemtype="http://schema.org/Article">
  <link itemprop="mainEntityOfPage" href="http://onesimple.cn/blog/2017/02/logistic-regression/">

  <span style="display:none" itemprop="author" itemscope itemtype="http://schema.org/Person">
    <meta itemprop="name" content="Gaobo Wang">
    <meta itemprop="description" content="">
    <meta itemprop="image" content="http://okqpu7sb4.bkt.clouddn.com/image/user/logo.png">
  </span>

  <span style="display:none" itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
    <meta itemprop="name" content="onesimple">
    <span style="display:none" itemprop="logo" itemscope itemtype="http://schema.org/ImageObject">
      <img style="display:none;" itemprop="url image" alt="onesimple" src="">
    </span>
  </span>

    
      <header class="post-header">

        
        
          <h1 class="post-title" itemprop="name headline">
            
            
              
                逻辑回归与线性回归原理探微
              
            
          </h1>
        

        <div class="post-meta">
          <span class="post-time">
            
              <time title="Post created" itemprop="dateCreated datePublished" datetime="2017-02-10T13:55:38+00:00">
                2017-02-10
              </time>
            

            

            
          </span>

          
            <span class="post-category" >
            
              <span class="post-meta-divider">|</span>
            
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              
                <span class="post-meta-item-text">分类于</span>
              
              
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/mechine-learning/" itemprop="url" rel="index">
                    <span itemprop="name">机器学习</span>
                  </a>
                </span>

                
                
              
            </span>
          

          
            
          

          

          
          
             <span id="/blog/2017/02/logistic-regression/" class="leancloud_visitors" data-flag-title="逻辑回归与线性回归原理探微">
               <span class="post-meta-divider">|</span>
               <span class="post-meta-item-icon">
                 <i class="fa fa-eye"></i>
               </span>
               
                 <span class="post-meta-item-text">阅读次数 </span>
               
                 <span class="leancloud-visitors-count"></span>
             </span>
          

          

          

        </div>
      </header>
    


    <div class="post-body" itemprop="articleBody">

      
      

      
        <center><br><img alt="封面" class="class1 class2" src="http://okqpu7sb4.bkt.clouddn.com//image/cover/ml.jpg?imageView2/2/w/600"><br></center>

<p>&emsp;&emsp;回归和分类是机器学习解决的最重要的问题，本篇主要整理了线性回归和逻辑回归的基本推理过程和工程实践应用，首先对两种回归的方法做一下比较:</p>
<blockquote>
<p>线性回归: 最小二乘法推导，预测连续值</p>
<p>逻辑回归: 基于最大似然估计，主要解决二分类问题，也可用于连续值的预测</p>
</blockquote>
<a id="more"></a>
<h1 id="1-线性回归"><a href="#1-线性回归" class="headerlink" title="1.线性回归"></a>1.线性回归</h1><h2 id="问题引出"><a href="#问题引出" class="headerlink" title="问题引出"></a>问题引出</h2><p>&emsp;&emsp;房价y和房屋面积x之间的关系，我们一般估计为线性关系，即目标函数为:</p>
<p>$$<br>    f(x) = ax + b<br>$$<br>$$<br>   f(x) \approx  y<br>$$<br>根据样本很容易计算出a、b，也就能根据一个给定的面积x估算出房价了。\<br>如果有影响房价的不止一个因素(现实中往往如此，比如国民平均收入水平也是影响房价的一个因素)，就是一个多变量问题，这时我们估计的目标函数就变成多元的了，如果有m个因素,记为列向量:<br>$$\sum_{i=1}^n a_i=0$$</p>
<p>$$<br>    x = (x_0,x_1,…,x_{m-1})<br>$$<br>则要求解的参数w的也应该是m个，也记为列向量<br>$$<br>    w =<br>        \left (<br>            \begin{matrix}<br>                w_0 \\<br>                w_1 \\<br>                … \\<br>                w_{m-1}<br>            \end{matrix}<br>        \right )<br>$$<br>预测的f(x)是一个值，<code>f(x) = ax +b</code>的形式就变成:<br>$$<br>    f = xw<br>$$<br>推广到样本空间，记第i个样本为:<br>$$<br>    x_i = (x_{i0}, x_{i1},…,x_{i,m-1})<br>$$<br>如果有n个样本，样本空间表示为矩阵X:<br>$$<br>    X_{n \times m} =<br>        \left (<br>            \begin{matrix}<br>                x_{00} &amp; x_{01} &amp; … &amp; x_{0,m-1} \\<br>                x_{10} &amp; x_{11} &amp; … &amp; x_{1,m-1} \\<br>                …    &amp; … &amp; … &amp; …          \\<br>                x_{n-1,0} &amp; x_{n-1,1} &amp; … &amp; x_{n-1,m-1}<br>            \end{matrix}<br>        \right )<br>$$<br>则预测的所有f值的向量为:<br>$$<br>    f_{n \times 1} =<br>            \left (<br>            \begin{matrix}<br>                f_0 \\<br>                f_1 \\<br>                … \\<br>                f_{n-1}<br>            \end{matrix}<br>        \right )<br>        =<br>        X_{n \times m}w_{m \times 1}<br>$$<br>预测值和实际值一般会有一个误差，记为e<br>$$<br>  y_{n \times 1} = f_{n \times 1} + e_{n \times 1}<br>$$<br>任意第i个样本跟预测结果的误差的分布是随机的，既然是随机的，根据中心极限定理就可以估计其分布为正态分布，均值为0，方差为定值，则依据正态分布的概率密度定义,预测值与实际值之间误差的概率密度为:</p>
<p>$$<br>  p({e^{(i)}}) = \frac    {1}<br>      {<br>           \sqrt[2]  {2\pi}<br>           \sigma<br>      }<br>      e^{( - \frac { ( {e^{(i)}} ) ^ 2}<br>          {2\sigma^2}<br>      )}<br>$$<br>其中<br>$$<br>{e^{(i)}} = {y^{(i)}} - w_{m \times 1}^T {x_{m \times 1}^{(i)}}<br>$$<br>此式代入上式。假定n个样本独立，则n个样本的最大似然函数是各自概率密度的乘积:<br>$$<br>L(w) = \prod_{i=1}^{n} \frac    {1}<br>      {<br>           \sqrt[2]  {2\pi}<br>           \sigma<br>      }<br>      e^{( - \frac { ( {e^{(i)}} ) ^ 2}<br>          {2\sigma^2}<br>      )}<br>$$<br>此式是关于w的似然函数，要求的是w，可以先对上式取对数,得:<br>$$<br>l(w) = \sum_{i=1}^{n} \ln  [ {<br>       \frac    {1}<br>      {<br>           \sqrt[2]  {2\pi}<br>           \sigma<br>      }<br>      e^{( - \frac { ( {e^{(i)}} ) ^ 2}<br>          {2\sigma^2}<br>      )}<br>      } ]    \\\\<br>     = n \ln (\frac {1} { \sqrt[2]  {2\pi} \sigma }) -<br>     \frac {1} {2\sigma^2} \sum_{i=1}^{n} ({y^{(i)}} - w_{n \times 1}^T {x_{n \times 1}^{(i)}})^2<br>$$<br>前半项是常数项，记后半项为:<br>$$ \begin{equation}<br>J(w) = \frac {1} {2} {\sum_{i=1}^{n} ({y^{(i)}} - w_{n \times 1}^T {x_{n \times 1}^{(i)}})^2} \\\\<br>     = \frac {1} {2} {\sum_{i=1}^{n} (f_w(x) - y)^2}<br>     \label{a0} \end{equation}<br>$$</p>
<blockquote>
<p>l(w) J(w) 都是一个数值，根据最大似然估计法，概率总和最大时参数估计的最合理，则最大似然函数L(w)取最大时，可推出J(w)取最小，J(w)为最小二乘法的目标函数，问题变为求目标函数最小值的问题。</p>
</blockquote>
<h2 id="参数学习"><a href="#参数学习" class="headerlink" title="参数学习"></a>参数学习</h2><p>X为nxm的样本矩阵，Y为nx1维列向量，J(w)可以写成以下形式:<br>$$<br>\begin{equation}<br>J(w) = \frac {1} {2} (Xw - Y)^T(Xw - Y)<br>\label{a1}<br>\end{equation}<br>$$<br>要求J(w)的极值，可对上式求偏导:<br>$$<br>\nabla _w J(w) = \nabla _w (Xw - Y)^T(Xw - Y) \\\\<br>= \nabla _w (w^T X^T X w - w^T X^T Y - Y^T X w - Y^T Y) \\\\<br>= 2 X^T X w - X^T Y - (Y^T X)^T \\\\<br>= 2 ( X^T X w - X^T Y )<br>$$</p>
<p>注意:</p>
<blockquote>
<p>$Y^T X$  是常数项对w求偏导为0<br>$ \frac{\partial Xw}{\partial w} = X^\mathrm {T} $</p>
</blockquote>
<p>令上式偏导等于0，求解w:<br>$$<br>w = ( X^\mathrm {T} X ) ^ {-1} X^\mathrm {T} Y<br>$$<br>为防止$ X^\mathrm {T} X $ 不可逆或者防止过拟合，增加扰动$\lambda $<br>$$<br>\begin{equation}<br>w = ( X^\mathrm {T} X + \lambda I ) ^ {-1} X^\mathrm {T} Y<br>\label{a2}<br>\end{equation}<br>$$<br>I为单位阵，以上就是线性回归数理上的最终结论。</p>
<h2 id="正则化处理"><a href="#正则化处理" class="headerlink" title="正则化处理"></a>正则化处理</h2><p>过拟合是指在样本集上表现良好，但测试集上不好，一般是w中有些值过大，振荡很大，而导致过拟合。<br>为解决这个问题可以让更多的$w_j$取0，且越多越好，这种处理方法叫$L_0$正则化；<br>一般是让$ \sum_{j=1}^{m} |w_j| $ 取最小，这种处理叫$L_0$正则化；<br>如果是让$ \sum_{j=1}^{m} |w_j|^2 $ 取最小，则称谓$L_2$正则化。<br>故目标函数 $(\ref{a1})$ 加入平方和损失:<br>$$<br>\begin{equation}<br>J(w) = \frac {1} {2} (Xw - Y)^T(Xw - Y) + \lambda \sum_{j=1}^{m} {w_j}^2<br>\end{equation}<br>$$<br>根据此式进行求解也能推出w的求解公式$(\ref{a2})$</p>
<h2 id="训练与评价"><a href="#训练与评价" class="headerlink" title="训练与评价"></a>训练与评价</h2><p>怎么对样本进行训练呢?对于正则化项$\lambda$,怎么取值合理呢？</p>
<p>|  <strong><code>------- 1 -------</code></strong>    | <strong><code>-- 2 --</code></strong> |<strong><code>-- 3 --</code></strong>|</p>
<p>按合适比例划分样本，得到1、2、3三个部分<br>区间1用于训练w<br>区间2可以试用不同的$\lambda$，使用$w+\lambda I $作预测，看$\lambda$取多少时预测的准确，则选定此$\lambda$，这个过程不需要严格。<br>区间3对w、$\lambda$ 进行验证，当然，还有些交叉验证的策略。<br>|  <strong><code>------------  训练 -------------</code></strong>    | <strong><code>10% 验证</code></strong> |<br>|  <strong><code>--------  训练 ---------</code></strong>    | <strong><code>10%验证</code></strong> | <strong><code>- 训练-</code></strong> |<br>… …       交叉训练验证走十次       … …</p>
<h2 id="梯度下降算法"><a href="#梯度下降算法" class="headerlink" title="梯度下降算法"></a>梯度下降算法</h2><p>公式$(\ref{a2})$是参数w的解析解,但实际工程中样本空间很大，不可能对样本空间进行矩阵运算，所以这就需要机器学习的思路，从参数迭代的角度来求解w,梯度下降是常用的算法。</p>
<blockquote>
<p>其意义是，为了求得函数的极值点(局部最小值)，先任取一点，然后沿此点处梯度的负方向进行迭代到一个新的点，继续下降直到收敛，算法结束。</p>
</blockquote>
<p>首先我们来对$(\ref{a0})$某个$w_j$求偏导<br>$$<br> \frac{\partial J(w)}{\partial w_j} = \frac{\partial }{\partial w_j} { \frac {1} {2} (f_w (x) - y)^2 } \\\\<br> = (f_w (x) - y) \cdot \frac{\partial }{\partial w_j} (f_w (x) - y)\\\\<br> = (f_w (x) - y) \cdot \frac{\partial }{\partial w_j} (\sum_{k=1}^{m} w_k x_k - y) \\\\<br> = (f_w (x) - y) x_j<br>$$<br>注意，上式除了$w_j$其他都算常量，所以求导得到上式结果。结论可简记为：</p>
<blockquote>
<p>某个维度的梯度 = 预测误差 X 样本在该维度的取值</p>
</blockquote>
<p>既然有了这个梯度，我们就可以有以下几种方式计算。</p>
<h3 id="批量梯度下降"><a href="#批量梯度下降" class="headerlink" title="批量梯度下降"></a>批量梯度下降</h3><p><strong>批量梯度下降是对所有样本的梯度做加和，然后按步长进行下降迭代，直到收敛</strong></p>
<p>while  重复直到收敛 :  {<br>$$ w_j = w_j + \alpha \sum_{i=1}^{n} [ (y^{(i)} - f_w(x^{(i)})) \cdot x_j^{(i)} ] $$<br>}<br>$\alpha$为每次迭代的步长<br>凸函数局部最小值一定是全局最小值，批量梯度下降一定会收敛。<br>初始可以让w都为0</p>
<h3 id="随机梯度下降SGD"><a href="#随机梯度下降SGD" class="headerlink" title="随机梯度下降SGD"></a>随机梯度下降SGD</h3><p>随机梯度下降也称SGD<br><strong>针对每来一个样本进行一次迭代，而不是对所有样本的梯度做加和做迭代</strong><br>while 迭代次数  :  {<br>&emsp;&emsp;for i = 0 to n : {<br>$$<br>w_j = w_j + \alpha (y^{(i)} - f_w(x^{(i)})) \cdot x_j^{(i)}<br>$$<br>&emsp;&emsp;}<br>}<br>注:可以通过最大迭代次数来退出loop，也可以判断本次参数$w_j^{(k)}$ 和下次的$w_j^{(k+1)}$之间的误差，如果稳定到很小的值，则认为收敛，可以退出。</p>
<blockquote>
<p>全局梯度下降有可能在一些拐点位置停止不前，而随机梯度下降就或许能跳过一些拐点到达真正的收敛点附近，这是梯度下降的一些特点，以后有机会附文研讨。<br>随机梯度下降一般速度会比较快，学习步长也可以让它先大后小。</p>
</blockquote>
<h3 id="mini-batch梯度下降"><a href="#mini-batch梯度下降" class="headerlink" title="mini-batch梯度下降"></a>mini-batch梯度下降</h3><p>基于以上两种算法的折中算法，每次随机选取b个样本做一个批量梯度下降，这样既节省了计算整个批量的时间，同时计算的方向也会像SGD一样更加准确。<br>repeat until convergency{<br>&emsp;&emsp;for i=1;i&lt;n ; i+=b: {<br>$$ w_j = w_j + \alpha \sum_{i=1}^{i+b} [ (y^{(i)} - f_w(x^{(i)})) \cdot x_j^{(i)} ] $$<br>&emsp;&emsp;}<br>}
　</p>
<h1 id="2-非线性回归"><a href="#2-非线性回归" class="headerlink" title="2.非线性回归"></a>2.非线性回归</h1><h2 id="区别"><a href="#区别" class="headerlink" title="区别"></a>区别</h2><p>其实只是特征增加了$x^2$项，y相对于特征是非线性的，相对于w还是线性的。往往可以先通过变量置换，$x^{‘} = x^2$把非线性回归化为线性回归，再利用线性回归的方法确定参数及估计值。</p>
<blockquote>
<p><strong>局部加权回归暂不介绍</strong></p>
</blockquote>
<h1 id="3-逻辑回归"><a href="#3-逻辑回归" class="headerlink" title="3.逻辑回归"></a>3.逻辑回归</h1><h2 id="问题引出-1"><a href="#问题引出-1" class="headerlink" title="问题引出"></a>问题引出</h2><p>线性回归适合预测连续值，在解决分类问题时则有些不合理，比如<br>如果只有A B 两簇，线性回归拟合直线为$L_1$,此时取定义域中点a处的值作为界限进行分类，可能暂时满足。但当是ABC三簇数据时，再依此法取b出的值作为分类界限，就会发现分类很不合理。所以线性回归解决分类问题是不合理的，而逻辑回归则适合解决二分类问题。(如果解决多分类问题，可使用softmax回归)</p>
<center><br><img title="线性回归拟合直线" alt="线性回归拟合直线" class="class1 class2" src="http://okqpu7sb4.bkt.clouddn.com//image/data/line-rg1.png?imageView2/2/w/600"><br></center>

<p>逻辑回归虽然叫回归，但它实际是一种分类方法，主要解决分类问题，它的核心是利用了Logistic函数（也称Sigmoid函数），函数形式为:</p>
<p>$$<br>f{(x)} = \frac {1} {1 + e^{-x}}<br>$$</p>
<center><br><img title="Sigmoid函数" class="class1 class2" src="http://okqpu7sb4.bkt.clouddn.com//image/data/sigmoid.jpg?imageView2/2/w/600"><br></center>

<p>逻辑回归中，取预测值$w^Tx$为自变量，得</p>
<p>$$<br>\begin{equation}<br>f_w{(x)} = g(w^Tx) = \frac {1} {1 + e^{-{w^T}x}}<br>\label{a3}<br>\end{equation}<br>$$</p>
<p>上式可得到(0,1)区间的的分类预测值，以0.5为界进行划分即可完成分类。<br>要求解参数w，我们先对Sigmoid函数求偏导:<br>$$<br>\begin{equation}<br>g^{‘}{(x)} = ( \frac {1} {1 + e^{-x}} )^{‘} \\\\<br>    = \frac {e^{-x}} {(1+e^{-x})^2} \\\\<br>    = \frac {1} {1 + e^{-x}} \cdot { \frac {e^{-x}} {1+e^{-x}} } \\\\<br>    = \frac {1} {1 + e^{-x}} \cdot { (1 - {\frac {1} {1+e^{-x}}}) } \\\\<br>    = g(x)(1-g(x))<br>\label{a4}<br>\end{equation}<br>$$</p>
<p>和线性回归高斯分布不同，逻辑回归是二项分布，可以写成分段概率形式：<br>$$<br>p(y=1|x,w) = f_w{(x)}<br>p(y=0|x,w) = 1 - f_w{(x)}<br>$$<br>整合一下可以写成:<br>$$<br>p(y|x,w) = (f_w{(x)})^y (1 - f_w{(x)})^{(1-y)}<br>$$<br>似然函数变为：<br>$$<br>L(w) = \prod_{i=1}^{n} p( y^{(i)} | x^{(i)} ; w)<br>$$<br>对数似然函数：<br>$$<br>l(w) = \sum_{i=1}^{n} {( y^{(i)}  \ln {(f_w{(x^{(i)}}))}  + (1-y^{(i)}) \ln {(1-f_w{(x^{(i)}})}    )}<br>$$</p>
<p>对$w_j$求偏导,并代入$(\ref{a3})$,$(\ref{a4})$，可得:<br>$$<br> \frac{\partial l(w)}{\partial w_j}<br> = (y- f_w (x)) x_j<br>$$</p>
<p>进而可以得到梯度上升（往往也通称梯度下降）的迭代公式为:</p>
<p>$$<br>w_j = w_j + \alpha (y^{(i)} - f_w(x^{(i)})) \cdot x_j^{(i)} + \lambda w_j<br>$$</p>
<p>预测时：<br>$$<br>f(x|w) &gt; 0.5 ,y = 1<br>f(x|w) &lt;= 0.5 ,y = 0<br>$$<br>这里的f(x|w)也可以理解为后验概率来用。</p>
<h1 id="4-总结"><a href="#4-总结" class="headerlink" title="4.总结"></a>4.总结</h1><p>逻辑回归是基于最大似然估计和二项分布的，跟线性回归中用的最小二乘法没有关系。<br>后续将补充一些逻辑回归做预测的应用实例，并介绍谷歌ftrl在线学习算法等，敬请关注！</p>

      
    </div>

    <div>
      
        

      
    </div>

    <div>
      
        
<div style="padding: 10px 0; margin: 20px auto; width: 90%; text-align: center">
  <div>🐶  五百年雨打的石桥，有你走过  🐶</div>
  <button id="rewardButton", disable="enable", onclick="var qr = document.getElementById('QR'); if (qr.style.display === 'none') {qr.style.display='block';} else {qr.style.display='none'}", style="cursor: pointer; border: 0; outline: 0; border-radius: 100%; padding: 0; margin: 0; letter-spacing: normal; text-transform: none; text-indent: 0px; text-shadow: none">
    <span onmouseover="this.style.color='rgb(236,96,0)';this.style.background='rgb(204,204,204)'" onMouseOut="this.style.color='#fff';this.style.background='rgb(236,96,0)'" style="display: inline-block; width: 70px; height: 70px; border-radius: 100%; line-height: 81px; color: #fff; font: 400 35px/75px 'microsofty'; background: rgb(236,96,0)">赏</span>
  </button>
  <div id="QR" style="display: none;">
    
      <div id="wechat" style="display: inline-block">
        <img id="wechat_qr" src="http://okqpu7sb4.bkt.clouddn.com/image/user/wechatpay2.jpg" alt="Gaobo Wang WeChat Pay" style="width: 200px; max-width: 100%; display: inline-block"/>
        <p>微信打赏</p>
      </div>
    
    
      <div id="alipay" style="display: inline-block">
        <img id="alipay_qr" src="http://okqpu7sb4.bkt.clouddn.com/image/user/alipay2.jpg" alt="Gaobo Wang Alipay" style="width: 200px; max-width: 100%; display: inline-block"/>
        <p>支付宝打赏</p>
      </div>
    
  </div>
<div>


      
    </div>


    <footer class="post-footer">
      
        <div class="post-tags">
          
            <a href="/tags/mechine-learning/" rel="tag"># 机器学习</a>
          
            <a href="/tags/LR/" rel="tag"># 逻辑回归</a>
          
        </div>
      

      
        <div class="post-nav">
          <div class="post-nav-next post-nav-item">
            
              <a href="/blog/2016/10/svm-1/" rel="next" title="SVM支持向量机原理简析">
                <i class="fa fa-chevron-left"></i> SVM支持向量机原理简析
              </a>
            
          </div>

          <span class="post-nav-divider"></span>

          <div class="post-nav-prev post-nav-item">
            
          </div>
        </div>
      

      
      
    </footer>
  </article>



    <div class="post-spread">
      
        
  <div class="bdsharebuttonbox">
    <a href="#" class="bds_weixin" data-cmd="weixin" title="分享到微信"></a>
    <a href="#" class="bds_tsina" data-cmd="tsina" title="分享到新浪微博"></a>
    <a href="#" class="bds_sqq" data-cmd="sqq" title="分享到QQ好友"></a>
    <a href="#" class="bds_douban" data-cmd="douban" title="分享到豆瓣网"></a>
    <a href="#" class="bds_qzone" data-cmd="qzone" title="分享到QQ空间"></a>
    <a href="#" class="bds_tieba" data-cmd="tieba" title="分享到百度贴吧"></a>
    <a href="#" class="bds_twi" data-cmd="twi" title="分享到Twitter"></a>
    <a href="#" class="bds_fbook" data-cmd="fbook" title="分享到Facebook"></a>
    <a href="#" class="bds_more" data-cmd="more"></a>
    <a class="bds_count" data-cmd="count"></a>
  </div>
  <script>
    window._bd_share_config = {
      "common": {
        "bdText": "",
        "bdMini": "2",
        "bdMiniList": false,
        "bdPic": ""
      },
      "share": {
        "bdSize": "32",
        "bdStyle": "0"
      },
      "image": {
        "viewList": ["tsina", "douban", "sqq", "qzone", "weixin", "twi", "fbook"],
        "viewText": "分享到：",
        "viewSize": "32"
      }
    }
  </script>

<script>
  with(document)0[(getElementsByTagName('head')[0]||body).appendChild(createElement('script')).src='/static/api/js/share.js?v=89860593.js?cdnversion='+~(-new Date()/36e5)];
</script>

      
    </div>
  </div>

          
          </div>
          


          
  <div class="comments" id="comments">
    
      <div id="SOHUCS" sid="逻辑回归与线性回归原理探微"></div>
      <script charset="utf-8" type="text/javascript" src="https://changyan.sohu.com/upload/changyan.js" ></script>
      <script type="text/javascript">
      window.changyan.api.config({
      appid: 'cyt7ErSap',
      conf: 'prod_5dd0034658b5f16ee475267660f18e19'
      });
      </script>
    
  </div>


        </div>
        
          
  
  <div class="sidebar-toggle">
    <div class="sidebar-toggle-line-wrap">
      <span class="sidebar-toggle-line sidebar-toggle-line-first"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-middle"></span>
      <span class="sidebar-toggle-line sidebar-toggle-line-last"></span>
    </div>
  </div>

  <aside id="sidebar" class="sidebar">
    <div class="sidebar-inner">

      

      
        <ul class="sidebar-nav motion-element">
          <li class="sidebar-nav-toc sidebar-nav-active" data-target="post-toc-wrap" >
            文章目录
          </li>
          <li class="sidebar-nav-overview" data-target="site-overview">
            站点概览
          </li>
        </ul>
      

      <section class="site-overview sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
          <img class="site-author-image" itemprop="image"
               src="http://okqpu7sb4.bkt.clouddn.com/image/user/logo.png"
               alt="Gaobo Wang" />
          <p class="site-author-name" itemprop="name">Gaobo Wang</p>
          <p class="site-description motion-element" itemprop="description">just one simple</p>
        </div>
        <nav class="site-state motion-element">
        
          
            <div class="site-state-item site-state-posts">
              <a href="/archives">
                <span class="site-state-item-count">11</span>
                <span class="site-state-item-name">日志</span>
              </a>
            </div>
          

          
            <div class="site-state-item site-state-categories">
              
                <span class="site-state-item-count">3</span>
                <span class="site-state-item-name">分类</span>
              
            </div>
          

          
            <div class="site-state-item site-state-tags">
              <a href="/tags">
                <span class="site-state-item-count">10</span>
                <span class="site-state-item-name">标签</span>
              </a>
            </div>
          

        </nav>

        

        <div class="links-of-author motion-element">
          
            
              <span class="links-of-author-item">
                <a href="https://github.com/applefly" target="_blank" title="GitHub">
                  
                    <i class="fa fa-fw fa-github"></i>
                  
                  GitHub
                </a>
              </span>
            
              <span class="links-of-author-item">
                <a href="http://www.zhihu.com/people/one_simple" target="_blank" title="zhihu">
                  
                    <i class="fa fa-fw fa-weibo"></i>
                  
                  zhihu
                </a>
              </span>
            
          
        </div>

        
        

        
        

        


      </section>

      
      <!--noindex-->
        <section class="post-toc-wrap motion-element sidebar-panel sidebar-panel-active">
          <div class="post-toc">

            
              
            

            
              <div class="post-toc-content"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#1-线性回归"><span class="nav-number">1.</span> <span class="nav-text">1.线性回归</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#问题引出"><span class="nav-number">1.1.</span> <span class="nav-text">问题引出</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#参数学习"><span class="nav-number">1.2.</span> <span class="nav-text">参数学习</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#正则化处理"><span class="nav-number">1.3.</span> <span class="nav-text">正则化处理</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#训练与评价"><span class="nav-number">1.4.</span> <span class="nav-text">训练与评价</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#梯度下降算法"><span class="nav-number">1.5.</span> <span class="nav-text">梯度下降算法</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#批量梯度下降"><span class="nav-number">1.5.1.</span> <span class="nav-text">批量梯度下降</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#随机梯度下降SGD"><span class="nav-number">1.5.2.</span> <span class="nav-text">随机梯度下降SGD</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#mini-batch梯度下降"><span class="nav-number">1.5.3.</span> <span class="nav-text">mini-batch梯度下降</span></a></li></ol></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#2-非线性回归"><span class="nav-number">2.</span> <span class="nav-text">2.非线性回归</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#区别"><span class="nav-number">2.1.</span> <span class="nav-text">区别</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#3-逻辑回归"><span class="nav-number">3.</span> <span class="nav-text">3.逻辑回归</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#问题引出-1"><span class="nav-number">3.1.</span> <span class="nav-text">问题引出</span></a></li></ol></li><li class="nav-item nav-level-1"><a class="nav-link" href="#4-总结"><span class="nav-number">4.</span> <span class="nav-text">4.总结</span></a></li></ol></div>
            

          </div>
        </section>
      <!--/noindex-->
      

    </div>
  </aside>


        
      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="footer-inner">
        
<script async src="https://dn-lbstatics.qbox.me/busuanzi/2.3/busuanzi.pure.mini.js">
</script>

<div class="copyright" >
  
  &copy;  2017 - 
  <span itemprop="copyrightYear">2018</span>
  <span class="with-love">
    <i class="fa fa-heart"></i>
  </span>
  <span class="author" itemprop="copyrightHolder"> </span>

  
  <div class="powered-by">
      onesimple 
  </div>
  <span id="busuanzi_container_site_uv">
    <i class="fa fa-user"></i>
    <span id="busuanzi_value_site_uv"></span>
    <span>&nbsp;&nbsp;&nbsp;|&nbsp;&nbsp;&nbsp;</span>
  </span>

  <span id="busuanzi_container_site_pv">
      <i class="fa fa-eye"></i>
      <span id="busuanzi_value_site_pv"></span>
  </span>

</div>

<div>
  <a target="_blank" href="http://www.beian.gov.cn/portal/registerSystemInfo?recordcode=11010502036759" 
  style="display:inline-block;text-decoration:none;line-height:15px;">
  <img src="https://s1.ax1x.com/2018/09/29/ilmwIH.png" style="float:left;margin: 7px 0px 0px 5px;"/>
  <p style="float:left;height:20px;line-height:20px;margin: 7px 0px 0px 5px;10502036759号</p></a> color:#939393;">京公网安备 11010502036759号</p></a>
</div>

<p>Hosted by <a href="https://pages.coding.me" >Coding Pages</a></p>



        

        
      </div>
    </footer>

    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
    </div>
  </div>

  

<script type="text/javascript">
  if (Object.prototype.toString.call(window.Promise) !== '[object Function]') {
    window.Promise = null;
  }
</script>









  




  
  <script type="text/javascript" src="/lib/jquery/index.js?v=2.1.3"></script>

  
  <script type="text/javascript" src="/lib/fastclick/lib/fastclick.min.js?v=1.0.6"></script>

  
  <script type="text/javascript" src="/lib/jquery_lazyload/jquery.lazyload.js?v=1.9.7"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/velocity/velocity.ui.min.js?v=1.2.1"></script>

  
  <script type="text/javascript" src="/lib/fancybox/source/jquery.fancybox.pack.js?v=2.1.5"></script>


  


  <script type="text/javascript" src="/js/src/utils.js?v=5.1.0"></script>

  <script type="text/javascript" src="/js/src/motion.js?v=5.1.0"></script>



  
  

  
  <script type="text/javascript" src="/js/src/scrollspy.js?v=5.1.0"></script>
<script type="text/javascript" src="/js/src/post-details.js?v=5.1.0"></script>



  


  <script type="text/javascript" src="/js/src/bootstrap.js?v=5.1.0"></script>



  



  




	





  





  

  
      <!-- UY BEGIN -->
      <script type="text/javascript" src="http://v2.uyan.cc/code/uyan.js?uid="></script>
      <!-- UY END -->
  




  
  
  <script type="text/javascript">
    // Popup Window;
    var isfetched = false;
    // Search DB path;
    var search_path = "search.xml";
    if (search_path.length == 0) {
      search_path = "search.xml";
    }
    var path = "/" + search_path;
    // monitor main search box;

    function proceedsearch() {
      $("body").append('<div class="popoverlay">').css('overflow', 'hidden');
      $('.popup').toggle();
    }
    // search function;
    var searchFunc = function(path, search_id, content_id) {
      'use strict';
      $.ajax({
        url: path,
        dataType: "xml",
        async: true,
        success: function( xmlResponse ) {
          // get the contents from search data
          isfetched = true;
          $('.popup').detach().appendTo('.header-inner');
          var datas = $( "entry", xmlResponse ).map(function() {
            return {
              title: $( "title", this ).text(),
              content: $("content",this).text(),
              url: $( "url" , this).text()
            };
          }).get();
          var $input = document.getElementById(search_id);
          var $resultContent = document.getElementById(content_id);
          $input.addEventListener('input', function(){
            var matchcounts = 0;
            var str='<ul class=\"search-result-list\">';
            var keywords = this.value.trim().toLowerCase().split(/[\s\-]+/);
            $resultContent.innerHTML = "";
            if (this.value.trim().length > 1) {
              // perform local searching
              datas.forEach(function(data) {
                var isMatch = false;
                var content_index = [];
                var data_title = data.title.trim().toLowerCase();
                var data_content = data.content.trim().replace(/<[^>]+>/g,"").toLowerCase();
                var data_url = decodeURIComponent(data.url);
                var index_title = -1;
                var index_content = -1;
                var first_occur = -1;
                // only match artiles with not empty titles and contents
                if(data_title != '') {
                  keywords.forEach(function(keyword, i) {
                    index_title = data_title.indexOf(keyword);
                    index_content = data_content.indexOf(keyword);
                    if( index_title >= 0 || index_content >= 0 ){
                      isMatch = true;
                      if (i == 0) {
                        first_occur = index_content;
                      }
                    }

                  });
                }
                // show search results
                if (isMatch) {
                  matchcounts += 1;
                  str += "<li><a href='"+ data_url +"' class='search-result-title'>"+ data_title +"</a>";
                  var content = data.content.trim().replace(/<[^>]+>/g,"");
                  if (first_occur >= 0) {
                    // cut out 100 characters
                    var start = first_occur - 20;
                    var end = first_occur + 80;
                    if(start < 0){
                      start = 0;
                    }
                    if(start == 0){
                      end = 50;
                    }
                    if(end > content.length){
                      end = content.length;
                    }
                    var match_content = content.substring(start, end);
                    // highlight all keywords
                    keywords.forEach(function(keyword){
                      var regS = new RegExp(keyword, "gi");
                      match_content = match_content.replace(regS, "<b class=\"search-keyword\">"+keyword+"</b>");
                    });

                    str += "<p class=\"search-result\">" + match_content +"...</p>"
                  }
                  str += "</li>";
                }
              })};
            str += "</ul>";
            if (matchcounts == 0) { str = '<div id="no-result"><i class="fa fa-frown-o fa-5x" /></div>' }
            if (keywords == "") { str = '<div id="no-result"><i class="fa fa-search fa-5x" /></div>' }
            $resultContent.innerHTML = str;
          });
          proceedsearch();
        }
      });}

    // handle and trigger popup window;
    $('.popup-trigger').click(function(e) {
      e.stopPropagation();
      if (isfetched == false) {
        searchFunc(path, 'local-search-input', 'local-search-result');
      } else {
        proceedsearch();
      };
    });

    $('.popup-btn-close').click(function(e){
      $('.popup').hide();
      $(".popoverlay").remove();
      $('body').css('overflow', '');
    });
    $('.popup').click(function(e){
      e.stopPropagation();
    });
  </script>


  

  

  
  <script src="https://cdn1.lncld.net/static/js/av-core-mini-0.6.1.js"></script>
  <script>AV.initialize("P5IKT36qLTgpIOt0Ol0cK4RN-gzGzoHsz", "SWI9thoVUEVPrUD6sxwJTM5C");</script>
  <script>
    function showTime(Counter) {
      var query = new AV.Query(Counter);
      var entries = [];
      var $visitors = $(".leancloud_visitors");

      $visitors.each(function () {
        entries.push( $(this).attr("id").trim() );
      });

      query.containedIn('url', entries);
      query.find()
        .done(function (results) {
          var COUNT_CONTAINER_REF = '.leancloud-visitors-count';

          if (results.length === 0) {
            $visitors.find(COUNT_CONTAINER_REF).text(0);
            return;
          }

          for (var i = 0; i < results.length; i++) {
            var item = results[i];
            var url = item.get('url');
            var time = item.get('time');
            var element = document.getElementById(url);

            $(element).find(COUNT_CONTAINER_REF).text(time);
          }
          for(var i = 0; i < entries.length; i++) {
            var url = entries[i];
            var element = document.getElementById(url);
            var countSpan = $(element).find(COUNT_CONTAINER_REF);
            if( countSpan.text() == '') {
              countSpan.text(0);
            }
          }
        })
        .fail(function (object, error) {
          console.log("Error: " + error.code + " " + error.message);
        });
    }

    function addCount(Counter) {
      var $visitors = $(".leancloud_visitors");
      var url = $visitors.attr('id').trim();
      var title = $visitors.attr('data-flag-title').trim();
      var query = new AV.Query(Counter);

      query.equalTo("url", url);
      query.find({
        success: function(results) {
          if (results.length > 0) {
            var counter = results[0];
            counter.fetchWhenSave(true);
            counter.increment("time");
            counter.save(null, {
              success: function(counter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(counter.get('time'));
              },
              error: function(counter, error) {
                console.log('Failed to save Visitor num, with error message: ' + error.message);
              }
            });
          } else {
            var newcounter = new Counter();
            /* Set ACL */
            var acl = new AV.ACL();
            acl.setPublicReadAccess(true);
            acl.setPublicWriteAccess(true);
            newcounter.setACL(acl);
            /* End Set ACL */
            newcounter.set("title", title);
            newcounter.set("url", url);
            newcounter.set("time", 1);
            newcounter.save(null, {
              success: function(newcounter) {
                var $element = $(document.getElementById(url));
                $element.find('.leancloud-visitors-count').text(newcounter.get('time'));
              },
              error: function(newcounter, error) {
                console.log('Failed to create');
              }
            });
          }
        },
        error: function(error) {
          console.log('Error:' + error.code + " " + error.message);
        }
      });
    }

    $(function() {
      var Counter = AV.Object.extend("Counter");
      if ($('.leancloud_visitors').length == 1) {
        addCount(Counter);
      } else if ($('.post-title-link').length > 1) {
        showTime(Counter);
      }
    });
  </script><!-- hexo-inject:begin --><!-- Begin: Injected MathJax -->
<script type="text/x-mathjax-config">
  MathJax.Hub.Config({"tex2jax":{"inlineMath":[["$","$"],["\\(","\\)"]],"skipTags":["script","noscript","style","textarea","pre","code"],"processEscapes":true},"TeX":{"equationNumbers":{"autoNumber":"AMS"}}});
</script>

<script type="text/x-mathjax-config">
  MathJax.Hub.Queue(function() {
    var all = MathJax.Hub.getAllJax(), i;
    for(i=0; i < all.length; i += 1) {
      all[i].SourceElement().parentNode.className += ' has-jax';
    }
  });
</script>

<script type="text/javascript" src="//cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
</script>
<!-- End: Injected MathJax -->
<!-- hexo-inject:end -->



  

  


</body>
</html>
